import os


BOT_NAME = 'image_spider'

SPIDER_MODULES = ['image_spider.spiders']
NEWSPIDER_MODULE = 'image_spider.spiders'

# 遵守robots.txt规则(设为False以允许爬取)
ROBOTSTXT_OBEY = False


FEED_EXPORT_ENCODING = 'utf-8'
DEFAULT_REQUEST_HEADERS = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',  # 指定接受中文
    'Referer': 'https://image.baidu.com',
}


LOG_LEVEL = 'INFO'  # 确保INFO级别及以上的日志会输出
# 可选：将日志写入文件，方便查看
LOG_FILE = 'pipeline_log.txt'

DOWNLOADER_MIDDLEWARES = {}  # 临时禁用所有中间件

# 设置用户代理，模拟浏览器访问
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36'

# 配置图片管道
ITEM_PIPELINES = {
    'image_spider.pipelines.CustomImagePipeline': 300,

}

# 图片存储路径
IMAGES_STORE = os.path.abspath("onepiece_images")  # 绝对路径更可靠

# 允许的图片扩展名
IMAGES_EXPIRES = 90  # 图片过期时间(天)
IMAGES_THUMBS = {
    'small': (50, 50),
    'big': (270, 270),
}

DOWNLOAD_DELAY = 3  # 延长延迟至3秒，降低被识别概率
CONCURRENT_REQUESTS = 1  # 单并发，模拟人工浏览